import logging
import re
from pyppeteer import launch
from pyppeteer.errors import TimeoutError
import asyncio

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s: %(message)s')
INDEX_URL = 'https://spa2.scrape.center/page/{page}'
TIMEOUT = 10
TOTAL_PAGE = 10
WINDOW_WIDTH, WINDOW_HEIGHT = 1366, 768
HEADLESS = True

# browser, tab = None, None
async def init():
    global browser, tab
    browser = await launch(executablePath='D:\chrome-win\chrome.exe',
                           headless=HEADLESS,
                           ignoreDefaultArgs=['--enable-automation'],
                           args=[f'--window-size={WINDOW_WIDTH},{WINDOW_HEIGHT}'])
    tab = await browser.newPage()
    await tab.setViewport({'width': WINDOW_WIDTH, 'height': WINDOW_HEIGHT})


# 定义通用的爬取方法
async def scrape_page(url, selector):
    logging.info('scraping %s ...', url)
    try:
        await tab.goto(url)
        await tab.waitForSelector(selector, options={
            'timeout': TIMEOUT * 1000
        })
    except TimeoutError:
        logging.error('error occurred while scraping %s', url, exc_info=True)

# 爬取列表页
async def scrape_index(page):
    url = INDEX_URL.format(page=page)
    await scrape_page(url, '.item .name')

# 解析列表页
async def parse_index():
    return await tab.querySelectorAllEval('.item .name', 'nodes => nodes.map(node => node.href)')


# 爬取详情页
async def scrape_detail(url):
    await scrape_page(url,'h2')

# 提取详情页信息
async def parse_detail():
    url = tab.url
    name = await tab.querySelectorEval('h2', 'node => node.innerText')
    categories = await tab.querySelectorAllEval('.categories button span', 'nodes => nodes.map(node => node.innerText)')
    cover = await tab.querySelectorEval('.cover', 'node => node.src')
    score = await tab.querySelectorEval('.score', 'node => node.innerText')
    drama = await tab.querySelectorEval('.drama p', 'node => node.innerText')
    if ':' in name:
       name = re.sub(r':', '_', name)
    return {
        'url': url,
        'name': name,
        'categories': categories,
        'cover': cover,
        'score': score,
        'drama': drama
    }


# 数据存储
from os import makedirs
from os.path import exists
import json
RESULTS_DIR = 'results'
exists(RESULTS_DIR) or makedirs(RESULTS_DIR)
def save_data(data):
    if all(data.values()): # 确保所有数据项搜非空
        name = data.get('name')
        data_path = f'{RESULTS_DIR}/{name}.json'
        with open(data_path,'w',encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
    else:
        logging.error('Incomplete data, not saving %s', data)
async def main():
    await init()
    try:
        for page in range(1, TOTAL_PAGE + 1):
            await scrape_index(page)
            detail_urls =await parse_index()
            for detail_url in detail_urls:
                await scrape_detail(detail_url)
                detail_data = await parse_detail()
                logging.info('data %s', detail_data)
                save_data(detail_data)
                logging.info('Saving Successfully')
    finally:
        await browser.close()

if __name__ == '__main__':
    asyncio.run(main())
